# -*- coding=utf-8 -*-

'''
需要模块：
pip install you-get
pip install ffmpeg
you-get使用方式：
    you-get -o c:/e/temp/video/ https://www.bilibili.com/.../.../
    https://www.ixigua.com/6935334863411085349?wid_try=1
ffmeg合并音频与视频
  ffmpeg -i jikeyundong-video.mp4 -i jikeyundong-bitrate130315.mp4 -c:v copy -c:a aac -strict experimental 合成的新文件名.mp4
  TIPS:视频在前 音频在后
content.txt文件里内容示例：每一个url独占一行
https://www.bilibili.com/xxxxxx
https://www.ixigua.com/69090075
'''
import subprocess,base64,requests,os,time,shutil

PATH_ROOT = 'C:/e/temp/pic/'
VIDEO_SAVE_PATH = 'c:/e/temp/video/'
YOUGET_COMMAND_PATTERN = 'you-get --no-caption -o %s %s'
FFMPEG_COMMAND_PATTERN = 'ffmpeg -i %s -i %s -c:v copy -c:a aac -strict experimental %s -loglevel quiet'


#获取要下载视频的url列表list
def getVideoUrlList():
    urlList = []
    with open(r'content.txt','r') as urlFile:
        lines = urlFile.readlines()
        for _l in lines:
            #if '?' in _l :
            #    _l = _l[:_l.index('?')]
            urlList.append(_l.replace('\n','').replace('\r',''))
    return urlList

#用you-get下载视频 
#当前已验证可行的网站：
#bilibili.com
def youGetDownloadVideoByUrlList(urlList):
    for _url in urlList:
        subprocess.call(YOUGET_COMMAND_PATTERN % (VIDEO_SAVE_PATH,_url))


HEADER = {
    "user-agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 11_1_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/87.0.4280.88 Safari/537.36"
    #,"sec-ch-ua": 'Google Chrome";v="87", " Not;A Brand";v="99", "Chromium";v="87'
    #,"Referer": "https://www.ixigua.com"
}

from selenium import webdriver
from selenium.webdriver.chrome.service import Service
from selenium.webdriver.chrome.options import Options
#返回网页源文件html
current_url = '' #该变量来保存当前下载视频播放页面的url 来判断视频是否为视频集合列表中的一个 目前需要在getUrlsDictByHtmlForIxiguaCom方法中单独处理
def getVideoPageHtml(url:str):
    global current_url 
    current_url = url
    html = ''
    #------------------------- selenium begin -------------------------#
    # 使用headless无界面浏览器模式
    chrome_options = Options()
    chrome_options.add_argument('--headless') #增加无界面选项
    chrome_options.add_argument('--disable-gpu') #如果不加这个选项，有时定位会出现问题
    chrome_options.add_experimental_option('excludeSwitches',['enable-logging','enable-automation'])
    # 启动浏览器，获取网页源代码 chromedriver.exe放在phantomjs的bin目录下一起加入path变量
    serviceDriver = Service(executable_path='chromedriver.exe')
    browser = webdriver.Chrome(service=serviceDriver,options=chrome_options) 
    browser.get(url)
    time.sleep(3)
    html = browser.page_source
    browser.quit()
    return html
    #------------------------- selenium end -------------------------#

RESOLUTION = {'360p':1,'480p':2,'720p':3,'1080p':4,'1080p 60fps':5,'4k':6,'4k 30fps':7,'4k 60fps':8}
from bs4 import BeautifulSoup
import re,json
#解析页面得到音频与视频url字典集合
#当前可得到 ixigua.com 网站的无水印视频
#update time：2022-04-17
def getUrlsDictByHtmlForIxiguaCom(html:str):
    video_info_dict = {}
    #bsp = BeautifulSoup(html,'html.parser')
    #target_content = bsp.find(id='SSR_HYDRATED_DATA')
    #封面
    thumbnailUrl = re.findall('<meta content="(.+?)" itemprop="Thumbnail">',html)
    if len(thumbnailUrl) > 0 :
        base_pic = thumbnailUrl[0]
        backup_pic = re.sub('tplv-.+?-crop','tplv-crop',base_pic)
        backup_pic = re.sub(':\d+:\d+\.',':0:0.',backup_pic)
        backup_pic = re.sub('\?.*$','',backup_pic)
        backup_pic = re.sub('^http[s]{0,1}:\/\/.+?\..+?\..+?\/','https://p3-xg.byteimg.com/',backup_pic)
        video_info_dict['thumbnailUrlBasePic'] = base_pic
        video_info_dict['thumbnailUrlBackupPic'] = backup_pic
    #无水印视频和音频
    json_content = re.findall('>window._SSR_HYDRATED_DATA=(.+?)</script>',html)
    if len(json_content) > 0 :
        dynamic_video_list = []
        dynamic_audio_list = []
        json_content = json_content[0].replace('\"','"').replace('\\"','"').replace(':undefined',':""').replace('"{','{').replace('}"','}')
        #------------begin------------------#
        #illegally_character_for_json = re.findall('\"video_abstract\":\".*?(\".*?\").*?\"',json_content)
        #for icj in illegally_character_for_json:
        #    json_content = re.sub(icj,icj.replace("\"","@@"),json_content)
        #------------end------------------#
        target_json_data = json.loads(json_content)
        video_title = target_json_data['anyVideo']['gidInformation']['packerData']['video']['title']
        if 'video_list' in target_json_data['anyVideo']['gidInformation']['packerData']['video']['videoResource']['dash']:
            dynamic_video_list = target_json_data['anyVideo']['gidInformation']['packerData']['video']['videoResource']['dash']['video_list']
        elif 'video_list' in target_json_data['anyVideo']['gidInformation']['packerData']['video']['videoResource']['normal']: 
            dynamic_video_list = target_json_data['anyVideo']['gidInformation']['packerData']['video']['videoResource']['normal']['video_list']
        else :
            dynamic_video = target_json_data['anyVideo']['gidInformation']['packerData']['video']['videoResource']['dash']['dynamic_video']
            dynamic_video_list = dynamic_video['dynamic_video_list']
            dynamic_audio_list = dynamic_video['dynamic_audio_list']

        video_dict = {}
        temp_video_dict = {}
        index = len(dynamic_video_list)
        for _video in dynamic_video_list:
            element_dict = {}
            if isinstance(_video,str):
                element_dict['definition']=dynamic_video_list[_video]['definition']
                element_dict['codec_type']=dynamic_video_list[_video]['codec_type']
                element_dict['size']=dynamic_video_list[_video]['size']
            #-----extend 20220808 begin-----#
                if 'id=' in current_url :
                    element_dict['main_url']=base64.b64decode(dynamic_video_list[_video]['backup_url_1']).decode('utf-8')
                else :
                    element_dict['main_url']=base64.b64decode(dynamic_video_list[_video]['main_url']).decode('utf-8')
            #-----extend 20220808 end-----#
            else :
                element_dict['definition']=_video['definition']
                element_dict['codec_type']=_video['codec_type']
                element_dict['size']=_video['size']
                if 'id=' in current_url :
                    element_dict['main_url']=base64.b64decode(_video['backup_url_1']).decode('utf-8')
                else :
                    element_dict['main_url']=base64.b64decode(_video['main_url']).decode('utf-8')
                #video_dict[str(_video['definition']).replace(' ','-')] = element_dict
                #video_dict[element_dict['definition']] = element_dict #以视频分辨率为key
                #video_dict[index] = element_dict
            temp_video_dict[RESOLUTION[element_dict['definition']]] = element_dict
            index -= 1
        temp_video_dict = sortedForDict(temp_video_dict,sorttype='dec')
        no = 1
        for k,v in temp_video_dict:
            video_dict[no]=v
            no += 1

        audio_dict = {}
        temp_audio_dict = {}
        index = len(dynamic_audio_list)
        for _audio in dynamic_audio_list:
            #raise ValueError("代码for _audio in dynamic_audio_list主动抛了一个异常:需要处理一下dynamic_audio_list中audio的顺序 参照函数getUrlsDictByHtmlForIxiguaCom中的temp_video_dict")
            element_dict = {}
            element_dict['bitrate']=_audio['bitrate']
            element_dict['codec_type']=_audio['codec_type']
            element_dict['main_url']=base64.b64decode(_audio['main_url']).decode('utf-8')
            #audio_dict[str(_audio['bitrate'])] = element_dict
            audio_dict[index] = element_dict
            index -= 1
        audio_dict = sortedForDict(audio_dict)

        video_info_dict['title']=delSpecialMarkOfTitle(video_title)
        video_info_dict['video']=video_dict
        video_info_dict['audio']=audio_dict
    return video_info_dict

#解析页面得到音频与视频url字典集合
#当前可得到 toutiao.com 网站的无水印视频
#update time：2022-09-08
from urllib.parse import quote,unquote
def getUrlsDictByHtmlForToutiaoCom(html:str):
    raw_text = re.findall('<script id="RENDER_DATA" type="application/json">(.*?)</script>',html)
    if len(raw_text) < 1 :
         raise ValueError("getUrlsDictByHtmlForToutiaoCom未找到有效信息！")
         exit()
    html=unquote(raw_text[0],'utf-8')
    #with open('./toutiao.json','w') as f:
    #    f.write(html)
    #    exit()
    video_info_dict = {}
    target_json_data = json.loads(html)
    video_title = target_json_data['data']['initialVideo']['title'].replace('#','')
    #封面
    thumbnailUrl = 'https:' + target_json_data['data']['initialVideo']['coverUrl']
    video_play_info = target_json_data['data']['initialVideo']['videoPlayInfo']
    dynamic_video_list = []
    if 'dynamic_video' in video_play_info :
        dynamic_video_list = video_play_info['dynamic_video']['dynamic_video_list']
    elif 'video_list' in video_play_info :
        dynamic_video_list = video_play_info['video_list']
    video_dict = {}
    temp_video_dict = {}
    for _video in dynamic_video_list :
        element_dict = {}
        rate = _video['video_meta']['definition']
        element_dict['main_url'] = _video['main_url']
        element_dict['definition'] = rate
        temp_video_dict[RESOLUTION[rate]] = element_dict
    temp_video_dict = sortedForDict(temp_video_dict,sorttype='dec')
    no = 1
    for k,v in temp_video_dict:
        video_dict[no]=v
        no += 1
    dynamic_audio_list = []
    if 'dynamic_video' in video_play_info :
        dynamic_audio_list = video_play_info['dynamic_video']['dynamic_audio_list']
    audio_dict = {}
    temp_audio_dict = {}
    index = len(dynamic_audio_list)
    for _audio in dynamic_audio_list :
        element_dict = {}
        element_dict['bitrate'] = _audio['audio_meta']['bitrate']
        element_dict['codec_type'] = _audio['audio_meta']['codec_type']
        element_dict['main_url'] = _audio['main_url']
        audio_dict[index] = element_dict
        index -= 1
    audio_dict = sortedForDict(audio_dict)

    video_info_dict['title'] = delSpecialMarkOfTitle(video_title)
    video_info_dict['thumbnailUrlBasePic'] = thumbnailUrl
    video_info_dict['thumbnailUrlBackupPic'] = thumbnailUrl
    video_info_dict['video'] = video_dict
    video_info_dict['audio'] = audio_dict

    #import pprint
    #pprint.pprint(video_info_dict)
    return video_info_dict



#解析页面得到音频与视频url字典集合
#当前可得到 bilibili.com 网站的视频
#update time：2022-04-18 (暂未发现B站无水印视频url 暂停开发)
def getUrlsDictByHtmlForBilibiliCom(html:str):
    video_info_dict = {}
    #bsp = BeautifulSoup(html,'html.parser')
    #封面
    thumbnailUrl = re.findall('<meta data-vue-meta="true" itemprop="thumbnailUrl" content="(.+?)">',html)
    if len(thumbnailUrl) > 0 :
        video_info_dict['thumbnailUrlBasePic'] = thumbnailUrl[0]
        video_info_dict['thumbnailUrlBackupPic'] = video_info_dict['thumbnailUrlBasePic']
    #标题
    video_title = re.findall('<h1 id="video-title" title="(.+?)" class="video-title">',html)
    if len(video_title) > 0 :
        video_info_dict['title'] = delSpecialMarkOfTitle(video_title[0])
    #视频和音频
    #json_content = re.findall('<script>window.__playinfo__=(.+?)</script>',html)
    #if len(json_content)> 0 :
    #    json_content = json_content[0]

    #print(video_info_dict)
    return video_info_dict

#解析页面得到音频与视频url字典集合
#当前可得到 youtube.com 网站的视频
#update time：2022-04-18 (开发中……)
def getUrlsDictByHtmlForYoutubeCom(html:str):
    #bsp = BeautifulSoup(html,'html.parser')
    json_content = re.findall('var ytInitialPlayerResponse = (.+?)}}}};',html)
    if len(json_content) > 0 :
        json_content = json_content[0] + '}}}};'

    print(json_content)

#给字典排序
def sortedForDict(_dict,sorttype='asc'):#默认升序既从小到大排序
    #按key从大到小
    if sorttype != 'asc' :
        return sorted(_dict.items(),key=lambda d:d[0],reverse=True)
    #按key从小到大
    return sorted(_dict.items(),key=lambda d:d[0])#默认reverse=False从小到大排序

#去掉不能在文件名中出现的和不想要的字符
def delSpecialMarkOfTitle(title:str):
    return re.sub(r'[\|:/<>?"* #\\]','',title)

#下载资源
def downloadByUrl(url,file_name):
    if not re.match('^http[s]{0,1}://',url):
        return
    if requests.head(url).status_code != 200:
        return
    _response = requests.get(url,headers=HEADER,stream=True)
    #_content = _response.content
    _content_size = int(_response.headers.get('content-length'))
    _data_count = 0
    with open(file_name,'ab') as _f :
        for chunk in _response.iter_content(chunk_size=40960):
            if chunk :
                _f.write(chunk)
                _data_count += len(chunk)
                _now_rate = (_data_count/_content_size)*100
                print('\rloading...: %02d%% (%d/%d)' % (_now_rate,_data_count,_content_size),end=' ')
    _response.close() #stream=True时必须要手动关闭response

#传入视频网页url下载视频
def getVideoFromWebPageUrl(url:str,func):
    page_html = getVideoPageHtml(url)
    video_info = func(page_html)
    #video_info = getUrlsDictByHtmlForIxiguaCom(page_html)
    #print(json.dumps(video_info,indent=4)) #为方便调试 修饰dict类型的输出格式
    #--------------下载资源 begin --------------#
    time_mark = time.strftime('%Y%m%d%H%M%S',time.localtime())

    video_file = os.path.join(PATH_ROOT,time_mark+'_video.mp4')
    if video_info.__contains__('video'):
        #-----extend 20220802 begin-----#
        _video = video_info['video'] if len(video_info['video'])>0 else None
        #-----extend 20220802 end-----#
        #_video = video_info['video'][0] if len(video_info['video'])>0 else None #'[0]'代表数组下标为0
        if _video is not None :
            downloadByUrl(_video[1]['main_url'],video_file) #'[1]'代表dict字典key为数字‘1’

    audio_file = PATH_ROOT+time_mark+'_audio.mp4'
    if video_info.__contains__('audio'):
        _audio = video_info['audio'][0] if len(video_info['audio'])>0 else None #'[0]'代表数组下标为0
        if _audio is not None :
            downloadByUrl(_audio[1]['main_url'],audio_file) #'[1]'代表dict字典key为数字‘1’

    video_title = video_info['title']
    if video_info.__contains__('thumbnailUrlBasePic'):
        thumbnail_url = video_info['thumbnailUrlBasePic']
        suffix = re.findall('\.jpg|\.jpeg|\.png|\.webp',thumbnail_url)
        suffix = suffix[0] if len(suffix)>0 else '.jpeg'
        thumbnail_file = PATH_ROOT+video_title+suffix
        #downloadByUrl(thumbnail_url,thumbnail_file.replace(suffix,'0'+suffix)) #thumbnailUrlBasePic是小图 不下载
        if video_info.__contains__('thumbnailUrlBackupPic'):
            downloadByUrl(video_info['thumbnailUrlBackupPic'],thumbnail_file)
    #--------------下载资源 end --------------#
    #--------------合并视频与音频 begin -------#
    if os.path.exists(video_file) and os.path.exists(audio_file):
        #print(FFMPEG_COMMAND_PATTERN % (video_file,audio_file,VIDEO_SAVE_PATH+video_title+'.mp4'))
        ret = subprocess.call(FFMPEG_COMMAND_PATTERN % (video_file,audio_file,VIDEO_SAVE_PATH+video_title+'.mp4'))
        if ret == 0 :
            #删除纯视频和纯音频文件
            os.remove(video_file)
            os.remove(audio_file)
        else :
            print('Oops! it`s some problem...:%s(%s)' % (video_title,url))
    #--------------合并视频与音频 end ---------#
    else :
        shutil.move(video_file,VIDEO_SAVE_PATH+video_title.replace(' ','')+'.mp4')
        #commondstr = 'xcopy %s %s' % (video_file,VIDEO_SAVE_PATH+video_title.replace(' ','')+'.mp4')
        #ret = subprocess.call(commondstr)
        #if ret == 0 :
        #    #删除视频文件
        #    #os.remove(video_file)
        #    print('Done:%s' % video_title)
        #else :
        #    print('Oops! it`s some problem...:%s(%s)' % (video_title,url))
    print('Done:%s;%s' % (video_title,url))

#主函数
#Tips:
# 1.待程序稳定后引入多线程
# 2.目前（2022-04-18）需要分别处理ixigua.com和bilibili.com的视频
def mainGetVideos():
    if not os.path.exists(PATH_ROOT):
        os.makedirs(PATH_ROOT)
    if not os.path.exists(VIDEO_SAVE_PATH):
        os.makedirs(VIDEO_SAVE_PATH)
    urlList = getVideoUrlList()
    current_url = ''
    try:
        for _url in urlList:
            current_url = _url#_url = re.sub('\?.*$','',_url)
            if re.match(r'^http[s]{0,1}://www.ixigua.com/',_url):
                getVideoFromWebPageUrl(_url,getUrlsDictByHtmlForIxiguaCom)
                continue
            if re.match(r'^http[s]{0,1}://www.toutiao.com/',_url):
                getVideoFromWebPageUrl(_url,getUrlsDictByHtmlForToutiaoCom)
                continue
            if re.match(r'^http[s]{0,1}://www.bilibili.com/',_url):
                #未发现B站无水印视频url数据 暂时只下载封面 (update 2022-04-19)
                getVideoFromWebPageUrl(_url,getUrlsDictByHtmlForBilibiliCom)
                #使用you-get下载B站视频 (不重复造轮)
                subprocess.call(YOUGET_COMMAND_PATTERN % (VIDEO_SAVE_PATH,_url))
    except Exception as e:
        print(repr(e))
        print('current_url: %s' % current_url)

if __name__ == '__main__':
    print('video download begin......')
    mainGetVideos()
    #---------------测试数据 调试代码用 保留 begin -----------------#
    #with open('fengmian.html','w',encoding='utf-8') as _f :
    #    html = getVideoPageHtml('https://www.ixigua.com/7068268141540803076')
    #    _f.write(html)
    #exit()
    #with open('bilibili.html','r',encoding='utf-8') as _f :
    #    html = _f.read()
    #    video_info = getUrlsDictByHtmlForBilibiliCom(html)
    #---------------测试数据 调试代码用 保留 end -------------------#
    print('video download end.')
